# -*- coding: utf-8 -*-
import scrapy

import logging
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem
import re

logger = logging.getLogger(__name__)


class AnqingSpider(scrapy.Spider):
    name = 'anqing'
    allowed_domains = ['aqhouse.net']
    start_urls = ['http://www.aqhouse.net/permitlist.aspx']
    custom_settings = {
        'DOWNLOAD_TIMEOUT': 120,
        'RETRY_TIMES': 30,
    }

    def parse(self, response):
        """
        提取参数，构造请求
        :param response:
        :return:
        """
        try:
            # 提取参数
            view_state = response.xpath("//input[@id='__VIEWSTATE']/@value").extract_first()
            view_state_generator = response.xpath("//input[@id='__VIEWSTATEGENERATOR']/@value").extract_first()
            event_validation = response.xpath("//input[@id='__EVENTVALIDATION']/@value").extract_first()
            assert view_state and view_state_generator and event_validation, '提取请求参数失败'
        except AssertionError as e:
            logger.error(e)
        except Exception as e:
            logger.error(f'提取请求参数失败，error：{e}')
        else:
            # 构造POST请求，获取预售许可证列表
            data: dict = {
                '__VIEWSTATE': view_state,
                '__VIEWSTATEGENERATOR': view_state_generator,
                '__EVENTVALIDATION': event_validation,
                'ddlMonth': 'no',
                'txtBUILDNAME': '',
                'txtSPNO': '',
                'txtCONAME': '公司',
                'btnFind': '按条件查询',
            }
            yield scrapy.FormRequest(
                self.start_urls[0],
                formdata=data,
                callback=self.parse_permit_li,
            )

    def parse_permit_li(self, response):
        """
        获取预售许可证列表
        :param response:
        :return:
        """
        try:
            permit_table_li = response.xpath("//span[@id='lblSalepermit']/table")
            assert permit_table_li, '预售许可证列表获取为空'
        except AssertionError as e:
            logger.error(e)
        else:
            for permit_table in permit_table_li:
                item_eg = FdcEstateGuidItem()
                item_eg['preSalePermit'] = permit_table.xpath(".//tr[1]/td[4]/text()").extract_first()  # 预售许可证
                item_eg['projectAddress'] = permit_table.xpath(".//tr[2]/td[2]/text()").extract_first()  # 项目地址
                item_eg['projectName'] = permit_table.xpath(".//tr[3]/td[2]/a/text()").extract_first()  # 项目名称
                project_url = permit_table.xpath(".//tr[3]/td[2]/a/@href").extract_first()
                item_eg['preSaleArea'] = permit_table.xpath(".//tr[3]/td[4]/text()").extract_first()  # 预售面积
                item_eg['roomUse'] = permit_table.xpath(".//tr[4]/td[2]/text()").extract_first()  # 房屋用途
                item_eg['preSaleRoomNo'] = permit_table.xpath(".//tr[4]/td[4]/text()").extract_first()  # 预售套数
                item_eg['landPermit'] = permit_table.xpath(".//tr[5]/td[2]/text()").extract_first()  # 土地许可证
                item_eg['planningPermit'] = permit_table.xpath(".//tr[6]/td[2]/text()").extract_first()  # 规划许可证
                land_usage_term_str = permit_table.xpath(".//tr[7]/td[2]/text()").extract_first()
                item_eg['landStartingDate'], item_eg['landTerminalDate'], item_eg[
                    'landUsageTerm'] = self.deal_land_usage_term(land_usage_term_str)  # 土地起始日期、土地终止日期、土地使用年限
                item_eg['preSaleBlockName'] = permit_table.xpath(".//tr[7]/td[4]/a/text()").extract_first()  # 预售楼栋名称
                item_eg['districtName'] = permit_table.xpath(".//tr[8]/td[2]/text()").extract_first()  # 项目区域
                item_eg['openingDate'] = permit_table.xpath(".//tr[8]/td[4]/text()").extract_first()  # 开盘日期
                if project_url:
                    item_eg['projectId'] = project_url.split('?')[1].split('&')[0].split('=')[1]  # 项目id
                    item_eg['permitId'] = project_url.split('?')[1].split('&')[1].split('=')[1]  # 许可证id
                    # 获取项目详情
                    yield response.follow(
                        project_url,
                        callback=self.parse_project_detail,
                        meta=dict(item_eg=deepcopy(item_eg), ),
                        priority=10,
                    )
                else:
                    logger.error('{}-{} 项目url提取为空'.format(item_eg['projectName'], item_eg['preSalePermit']))

    def parse_project_detail(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['developerName'] = response.xpath("//a[@id='hlkCompany']/text()").extract_first()  # 开发商
        item_eg['landGrade'] = response.xpath("//input[@id='txtLandlevel']/@value").extract_first()  # 土地等级
        item_eg['siteArea'] = response.xpath("//input[@id='txtLandarea']/@value").extract_first()  # 用地面积
        item_eg['estateTotalArea'] = response.xpath("//input[@id='txtBuildarea']/@value").extract_first()  # 建筑总面积
        item_eg['totalRoomNo'] = response.xpath("//span[@id='lblSets']/text()").extract_first()  # 总套数
        item_eg['soldRoomNo'] = response.xpath("//span[@id='lblSaledsets']/text()").extract_first()  # 已售套数
        item_eg['soldArea'] = response.xpath("//span[@id='lblSaledarea']/text()").extract_first()  # 已售面积
        item_eg['avgPrice'] = response.xpath("//span[@id='lblPrice']/text()").extract_first()  # 均价
        yield item_eg

        room_li_url = response.xpath(
            "//td[contains(text(),'{}')]/../td[1]/a/@href".format(item_eg['preSalePermit'])).extract_first()  # 房号列表url
        building_url = response.xpath(
            "//td[contains(text(),'{}')]/../td[6]/a/@href".format(item_eg['preSalePermit'])).extract_first()  # 楼栋url
        if room_li_url and building_url:
            # 获取楼栋信息
            yield response.follow(
                building_url,
                callback=self.parse_building_detail,
                meta=dict(item_eg=deepcopy(item_eg), room_li_url=room_li_url),
                priority=20,
            )
        else:
            logger.error('{}-{} 楼栋url和房号列表url提取为空}'.format(item_eg['projectName'], item_eg['preSalePermit']))

    def parse_building_detail(self, response):
        """
        获取楼栋信息
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_bd = FdcBuildingItem()
        item_bd['projectName'] = item_eg['projectName']  # 项目名称
        item_bd['projectId'] = item_eg['projectId']  # 项目id
        item_bd['preSalePermit'] = item_eg['preSalePermit']  # 预售许可证
        item_bd['permitId'] = item_eg['permitId']  # 许可证id
        item_bd['openingDate'] = item_eg['openingDate']  # 开盘日期
        item_bd['blockName'] = item_eg['preSaleBlockName']  # 楼栋名字
        del item_eg

        # 提取楼栋信息
        building_tr = response.xpath("//td[contains(text(),'总套数')]/../following-sibling::tr")
        building_td = building_tr.xpath("./td")
        try:
            assert building_tr, '{}-{}-{} 楼栋信息提取为空'.format(item_bd['projectName'], item_bd['preSalePermit'],
                                                           item_bd['blockName'])
            assert len(building_td) == 6, '{}-{}-{} 楼栋信息table长度提取出错'.format(item_bd['projectName'],
                                                                            item_bd['preSalePermit'],
                                                                            item_bd['blockName'])
        except AssertionError as e:
            logger.warning(e)
            item_bd['totalRoomNo'] = None  # 总套数
            item_bd['saleableRoomNo'] = None  # 可售套数
            item_bd['soldRoomNo'] = None  # 已售套数
            item_bd['buildingSoldArea'] = None  # 楼栋已售面积
            item_bd['avgPrice'] = None  # 均价
        else:
            item_bd['totalRoomNo'] = building_tr.xpath("./td[1]/text()").extract_first()  # 总套数
            item_bd['saleableRoomNo'] = building_tr.xpath("./td[2]/text()").extract_first()  # 可售套数
            item_bd['soldRoomNo'] = building_tr.xpath("./td[4]/text()").extract_first()  # 已售套数
            item_bd['buildingSoldArea'] = building_tr.xpath("./td[5]/text()").extract_first()  # 楼栋已售面积
            item_bd['avgPrice'] = building_tr.xpath("./td[6]/text()").extract_first()  # 均价
        finally:
            # 获取房号列表
            room_li_url = copy(response.meta['room_li_url'])
            yield response.follow(
                room_li_url,
                callback=self.parse_room_li,
                meta=dict(item_bd=deepcopy(item_bd), ),
                priority=30,
            )

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        item_bd['buildingId'] = response.request.url.split('?')[1].split('&')[0].split('=')[1]  # 楼栋id

        floor_set = set()
        # 楼层分组
        floor_tr_li = response.xpath("//td[contains(text(),'室序号')]/../following-sibling::tr")
        if floor_tr_li:
            for floor_tr in floor_tr_li:
                floor_num_str = floor_tr.xpath("./td[1]/text()").extract_first()
                floor_num = self.get_floor_num(floor_num_str)
                floor_set.add(floor_num) if isinstance(floor_num, int) else False  # 用于总楼层提取
                # 该楼层下的房号分组
                room_td_li = floor_tr.xpath('./td[position()>1]')
                if room_td_li:
                    for room_td in room_td_li:
                        item_rm = FdcRoomItem()
                        item_rm['projectName'] = item_bd['projectName']  # 项目名称
                        item_rm['projectId'] = item_bd['projectId']  # 项目id
                        item_rm['preSalePermit'] = item_bd['preSalePermit']  # 预售许可证
                        item_rm['blockName'] = item_bd['blockName']  # 楼栋名称
                        item_rm['buildingId'] = item_bd['buildingId']  # 楼栋id
                        item_rm['roomNo'] = room_td.xpath("./a/text()").extract_first()  # 房号
                        item_rm['saleStatus'] = self.get_sale_status(
                            room_td.xpath("./@bgcolor").extract_first())  # 销售状态
                        room_url = room_td.xpath("./a/@href").extract_first()
                        # 页面存在房号url为空的情况
                        if room_url:
                            # 获取房号详情
                            yield response.follow(
                                room_url,
                                callback=self.parse_room_detail,
                                meta=dict(item_rm=deepcopy(item_rm), ),
                                priority=40,
                            )
                else:
                    logger.warning('【{}】 {}-{}-{}-{} 房号分组为空'.format(response.request.url, item_bd['projectName'],
                                                                    item_bd['preSalePermit'],
                                                                    item_bd['blockName'], floor_num_str))
        else:
            logger.error(
                '【{}】 {}-{}-{} 房号列表为空'.format(response.request.url, item_bd['projectName'], item_bd['preSalePermit'],
                                              item_bd['blockName']))

        # 楼栋总楼层
        item_bd['floorTotalNo'] = self.get_floor_total_num(floor_set)
        yield item_bd

    def parse_room_detail(self, response):
        """
        获取房号详请
        :param response:
        :return:
        """
        item_rm = copy(response.meta['item_rm'])
        item_rm['roomLocation'] = response.xpath(
            "//td[contains(text(),'房屋坐落')]/following-sibling::td[1]/text()").extract_first()  # 房屋坐落
        item_rm['buildingArea'] = response.xpath(
            "//td[contains(text(),'建筑面积')]/following-sibling::td[1]/text()").extract_first()  # 建筑面积
        item_rm['innerArea'] = response.xpath(
            "//td[contains(text(),'套内面积')]/following-sibling::td[1]/text()").extract_first()  # 套内面积
        item_rm['apportionmentArea'] = response.xpath(
            "//td[contains(text(),'分摊面积')]/following-sibling::td[1]/text()").extract_first()  # 分摊面积
        item_rm['roomUse'] = response.xpath(
            "//td[contains(text(),'用途')]/following-sibling::td[1]/text()").extract_first()  # 房屋用途
        item_rm['roomStructure'] = response.xpath(
            "//td[contains(text(),'套型')]/following-sibling::td[1]/text()").extract_first()  # 房屋套型
        item_rm['unitPrice'] = response.xpath(
            "//td[contains(text(),'参考价格')]/following-sibling::td[1]/text()").extract_first()  # 参考价格
        yield item_rm

    @staticmethod
    def deal_land_usage_term(_str):
        """
        提取土地使用年限信息
        :param _str:
        :return:
        """
        regex_1 = re.compile(r'\d{4}-\d{1,2}-\d{1,2}')
        regex_2 = re.compile(r'共(\d+)')
        try:
            assert _str
            ret_1 = regex_1.findall(_str)
            assert len(ret_1) == 2
            ret_2 = regex_2.findall(_str)
            assert len(ret_2) == 1
        except:
            logger.error(f'【{_str}】 土地使用年限信息提取出错')
            return None, None, _str
        else:
            return ret_1[0], ret_1[1], ret_2[0]

    @staticmethod
    def get_floor_num(_str):
        """
        提取楼层
        :param _str:
        :return:
        """
        regex = re.compile(r'-?\d+')
        try:
            ret = int(regex.findall(_str)[0])
            assert ret
        except:
            return _str
        else:
            return ret

    @staticmethod
    def get_floor_total_num(_set):
        """
        提取楼栋总楼层
        :param _set:
        :return:
        """
        try:
            assert len(_set)
            floor_list = sorted(list(_set), reverse=True)
            ret = abs(floor_list[0])
        except:
            return
        else:
            return ret

    @staticmethod
    def get_sale_status(_str):
        """
        提取销售状态
        :param _str:
        :return:
        """
        status_dict = {
            '#cccccc': '保留',
            '#B0C4DE': '可售',
            '#ee8faa': '已租赁',
            '#ffff00': '已预订',
            '#ff9933': '已签约',
            '#0000ff': '已备案',
            '#00ff00': '已办产权证',
            '#ff0000': '被限制',
        }
        return status_dict.get(_str, None)
